package com.chance.cc.crawler.prod.command.job.domain.news.zhihu;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerJob;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.filter.FilterInfo;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.meta.core.bean.CrawlerMetaConstant;
import com.chance.cc.crawler.meta.core.bean.job.CrawlerScheduleJob;
import com.chance.cc.crawler.prod.command.job.domain.news.NewsCommonScript;
import org.apache.commons.lang3.StringUtils;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;

/**
 * created by CC on 2020/11/26
 * mail 279020185@qq.com
 */
public class ZHSearchTraceAHourCrawlerScheduleJob extends NewsCommonScript {

    public static final String domain = "zhihu";

    public static void main(String[] args) {
        publishCrawlerScheduleJobInfo();
    }

    public static CrawlerJob publishCrawlerScheduleJobInfo(){


        CrawlerJob jdCrawlerJob = crawlerJob();

        //发布定时采集作业
        CrawlerScheduleJob crawlerScheduleJob = new CrawlerScheduleJob();
        crawlerScheduleJob.setDomain(domain);
        crawlerScheduleJob.setCrawlerJob(JSON.toJSONString(jdCrawlerJob));
        crawlerScheduleJob.setJobType(CrawlerMetaConstant.ScheduleCrawlerJobType.crawler.enumVal());
        crawlerScheduleJob.setNote("知乎搜索关键词一小时追溯定时采集");
        crawlerScheduleJob.setCrawlerKey(jdCrawlerJob.generateCrawlerKey());
        HttpPage page = metaServiceCommand.addOrUpdateCrawlerScheduleJob(crawlerScheduleJob);
        System.out.println("发布作业：" + page.getRawText());
        return jdCrawlerJob;

    }

    /**
     * 18019432723
     * chancedata123
     * chance6
     * taobao采集
     * @return
     */
    public static CrawlerJob crawlerJob() {

        FilterInfo filterInfo = new FilterInfo();
        filterInfo.setFilter(CrawlerEnum.CrawlerRecordFilter.count);
        filterInfo.setCountTotalNum(100);

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpUrl("https://www.zhihu.com/api/v4/search_v3")
                .httpHead("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36")
                .httpHead("cookie","_xsrf=ncL2yiw14iTRyNfjjV3iBqJzxxYsE59B; _zap=57f60c38-b10d-478a-a6bd-93c9d8341a40; d_c0=\"AEAdHvQIGBOPTsjH-1x8nyug4onEvO8SfuE=|1620810474\"; __guid=74140564.372846385670046850.1620810474552.742; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1620810477; captcha_session_v2=\"2|1:0|10:1620810480|18:captcha_session_v2|88:ZngvbHowQTZ6WHg4UWgyc3Vwb0YxTWM5eHRDK0h1bVRLWitnWVg5QWVyeThwL2FVZlVJUG5pdHJIU3hVdGoreQ==|67c4d0eb2c2ab0a70089fe8464d6c46358515e4e779e74f1a752ae1bd78d3193\"; gdxidpyhxdE=ub7d7odoOn2cT%2Blbd%2BqXBhrCjPSm%2FjyeAiI7J%2FRz9sMkOZnCP%2B%5CMw7t9mJ9pSx1%2FpKarZsBbcA9INSGliSueqJ0h4gTVSWD4m5k%2BVjemQgk0QJ8ds0%5CZrjVw%5CTvMVHpDGXuH2ZWfVzqcujjm2ui3SQGBrZ%2BODcDhrfpjTJ1uZZfBCiQA%3A1620811381479; _9755xjdesxxd_=32; YD00517437729195%3AWM_NI=eSJGBcQCrBto1sEMS%2BlqTVutGVpDb1sK5XLi1QZsz17EAM6mZBqQ95gnctpnm%2BDcbvAXjD9fGkY7Ix%2FNlzXxqE8iKaaNNOpuuphrMpWUTanBLGGoZlClcrWyMWei%2B96ZY24%3D; YD00517437729195%3AWM_NIKE=9ca17ae2e6ffcda170e2e6eeaddc678ef0ff97fc68a5b48fb3d84b939f9e85b67df1bfbe94b253a99bf889f72af0fea7c3b92aa78d86a8c169939dbea3f765f3ad8baed221b0af9cb7f880858ac0a5e13bb195fbb9cf63f38f9694f93bf59b8faeb5679a9ebfb2bc3fa5b2ad95f366aff58c84e17e8388a6a5bc6e85b597d8cc74ed92a49bdb74b39bf88aca6ba39282b0f37fb7b0a893d07c958b9c82ed4a90998990d96881b4ad93d6738f89ab8aaa33f68697d3b737e2a3; YD00517437729195%3AWM_TID=40bMowKtE2JEBQFQQEJqwje75iMwE8go; captcha_ticket_v2=\"2|1:0|10:1620810493|17:captcha_ticket_v2|704:eyJ2YWxpZGF0ZSI6IkNOMzFfMWVNVlRZbWp1OFNUcXpwWldGQ05ZZW0yTnpRallWLjZHVnZhXy1JOHRURXJLVlFhTUEta2xBZG9lTG02Z281T3A0ajVENWhYNWtQNDBxLnA2bjdJeS1YeTI2bGplNGlfX3ROT0hjRUhzU2Z6ZVVBSVFXYWt5WUtmN3ZMZ0gySUdQQi1lUFN4SkJoOVlNQnJuNmZBV3dwbHpTenl3cGRRYXE1YmF6MFljUTdoVGdDS2ZRRVNmZ1hjeVlvT1lZUnBjV09US2dLVUNzai5YVzZ0SkNMTWhaUENVYmNUNGhEdkJuczgyWVEuNnZhLVg0LVJGbkZuR01iQXlBSHFFN25OVzhJNjhqQkJER3FYcUN0R2pYV29ZMFBDVVhkOGJNdnBZdmlQeWxaY0t3aktKOWhvU0xIUWJRcjZ4ZUpzYzdLV2xhTUtDRWpFLS1PUGZMYVFFUy5QMHZsRU5NekE0ZXRnNjZMRGh6V3djSmlqa0tGQXNOMDIyMHYyNTcuaGpkd2tHamVUQjIxZ0VfeUIxTXBIeGpGLmFGbnlnaThjNFNXZkN3NGV1S3VwNFJWNy0wOHZ1aXlpb2dxaVc0NTQ1aFR3dUc3aTB0dVJ4MTA0ZnQ1UEMyUUtMU2hCUnZGVGpaaGVadlBacFZyWjdoakJjNFI3dlN4dnZWOVp2eVdwMyJ9|c7122b279e208edd3e3a236fe17fdd09761e3f808a6d9a1f6a72dfeade67f386\"; z_c0=\"2|1:0|10:1620810493|4:z_c0|92:Mi4xWEpCYklBQUFBQUFBUUIwZTlBZ1lFeVlBQUFCZ0FsVk5fZWlJWVFBZVVmQkJHeWx3RUJ1WVgyalJGRUhWSUxpdC1n|c031538a2b3b1fd383bb55f6389590ea78e1005a0cfc0bd9529c75d74e9e53af\"; monitor_count=3; Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1620810500; KLBRSID=53650870f91603bc3193342a80cf198c|1620810501|1620810471")
                .releaseTime(System.currentTimeMillis())
                .needWashed(false)
                .needParsed(false)
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .turnPageFilterInfo(filterInfo)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(1,null))
                .proxy(proxy)
                .build();

        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);
        requestRecord.tagsCreator().bizTags().addSite("search");
        requestRecord.tagsCreator().bizTags().addDomain(domain);

        CrawlerRecord articleFilterCrawlerRecord = new CrawlerRecord();
        articleFilterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.dateRange);
        articleFilterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(1,null));
        requestRecord.tagsCreator().bizTags().addCustomKV("article_record_filter_info", JSON.toJSONString(articleFilterCrawlerRecord));

        CrawlerRecord commentFilterCrawlerRecord = new CrawlerRecord();
        commentFilterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.key);
        commentFilterCrawlerRecord.addFilterInfo(FilterUtils.redisFilterKeyInfo("filter-zhihu-search-comment-queue"));
        requestRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(commentFilterCrawlerRecord));

        requestRecord.tagsCreator().bizTags().addCustomKV("time_zone", "a_day");
        requestRecord.tagsCreator().bizTags().addCustomKV("scheduler_type", "a_hour");

        //初始 comment crawler request record
        String requestQueueName = StringUtils.joinWith("-","crawler",domain,"search","a_hour","queue");
        CrawlerRequestRecord initNewsCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("zhihu_news",turnPageItem)
                .httpUrl(metaServiceHttpPrefix+"/v1/meta/zhihu/keys?site=search")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        return CrawlerJob.builder()
                .crawlerJobThreadNumber(15)
                .triggerInfo(
                        domain,
                        CrawlerMetaConstant.ScheduleJobTrigger_Cron,
                        System.currentTimeMillis(),
                        StringUtils.joinWith("-","search","a","hour",CrawlerMetaConstant.ScheduleJobTriggerJob_Realtime))
                .crawlerRequestQueue(CrawlerMetaConstant.redisRequestQueue(requestQueueName))
                .kafkaResultPipeline(null,kafkaTopic,null)
                .fileResultPipeline("","/data/chance_crawler_runner/logs/node/zhihu-a_hour.log",false)
                .requestRecord(requestRecord)
                .supportRecord(initNewsCrawlerRecord)
                .build();
    }

}
